import scrapy

# 豆瓣爬虫 案例
class DoubanSpiderSpider(scrapy.Spider):
    name = "douban"
    allowed_domains = ["movie.douban.com"]
    start_urls = [
        "https://movie.douban.com/top250",
    ]

    custom_settings = {
        # 优先使用此设置覆盖全局 settings.py
        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'ROBOTSTXT_OBEY': False,
        'ITEM_PIPELINES': { },  # 禁用默认的 Item Pipeline；即无 数据结构定义和管道的处理逻辑

    }

    def start_requests(self):
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Referer': 'https://movie.douban.com/',
        }
        for url in self.start_urls:
            yield scrapy.Request(url, headers=headers, callback=self.parse)

    def parse(self, response):
        print("豆瓣电影爬虫开始运行！")

        # 提取电影信息
        for movie in response.css('div.item'):
            yield {
                'title': movie.css('span.title::text').get(),
                'rating': movie.css('span.rating_num::text').get(),
                'quote': movie.css('span.inq::text').get(default='无短评'),
            }

        # 处理分页
        next_page = response.css('span.next a::attr(href)').get()
        if next_page is not None:
            print(f"[DEBUG] 发现下一页: {next_page}")
            yield response.follow(next_page, callback=self.parse)


'''
在命令行中运行以下命令来启动爬虫：
scrapy crawl douban -o douban_movies.csv

'''
